{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 2: Reading and Writing Files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and writing CSV files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>invoice_no</th><th>customer_id</th><th>gender</th><th>age</th><th>category</th><th>quantity</th><th>price</th><th>payment_method</th><th>invoice_date</th><th>shopping_mall</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;I138884&quot;</td><td>&quot;C241288&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>&quot;5/8/2022&quot;</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>&quot;12/12/2021&quot;</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>&quot;9/11/2021&quot;</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>&quot;16/05/2021&quot;</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>&quot;24/10/2021&quot;</td><td>&quot;Kanyon&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌────────────┬─────────────┬────────┬─────┬───┬─────────┬──────────────┬─────────────┬─────────────┐\n",
       "│ invoice_no ┆ customer_id ┆ gender ┆ age ┆ … ┆ price   ┆ payment_meth ┆ invoice_dat ┆ shopping_ma │\n",
       "│ ---        ┆ ---         ┆ ---    ┆ --- ┆   ┆ ---     ┆ od           ┆ e           ┆ ll          │\n",
       "│ str        ┆ str         ┆ str    ┆ i64 ┆   ┆ f64     ┆ ---          ┆ ---         ┆ ---         │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆ str          ┆ str         ┆ str         │\n",
       "╞════════════╪═════════════╪════════╪═════╪═══╪═════════╪══════════════╪═════════════╪═════════════╡\n",
       "│ I138884    ┆ C241288     ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card  ┆ 5/8/2022    ┆ Kanyon      │\n",
       "│ I317333    ┆ C111565     ┆ Male   ┆ 21  ┆ … ┆ 1800.51 ┆ Debit Card   ┆ 12/12/2021  ┆ Forum       │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ Istanbul    │\n",
       "│ I127801    ┆ C266599     ┆ Male   ┆ 20  ┆ … ┆ 300.08  ┆ Cash         ┆ 9/11/2021   ┆ Metrocity   │\n",
       "│ I173702    ┆ C988172     ┆ Female ┆ 66  ┆ … ┆ 3000.85 ┆ Credit Card  ┆ 16/05/2021  ┆ Metropol    │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ AVM         │\n",
       "│ I337046    ┆ C189076     ┆ Female ┆ 53  ┆ … ┆ 60.6    ┆ Cash         ┆ 24/10/2021  ┆ Kanyon      │\n",
       "└────────────┴─────────────┴────────┴─────┴───┴─────────┴──────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_csv('../data/customer_shopping_data.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>I138884</th><th>C241288</th><th>Female</th><th>28</th><th>Clothing</th><th>5</th><th>1500.4</th><th>Credit Card</th><th>5/8/2022</th><th>Kanyon</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>&quot;12/12/2021&quot;</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>&quot;9/11/2021&quot;</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>&quot;16/05/2021&quot;</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>&quot;24/10/2021&quot;</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I227836&quot;</td><td>&quot;C657758&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>&quot;24/05/2022&quot;</td><td>&quot;Forum Istanbul…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌─────────┬─────────┬────────┬─────┬───┬─────────┬─────────────┬────────────┬────────────────┐\n",
       "│ I138884 ┆ C241288 ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card ┆ 5/8/2022   ┆ Kanyon         │\n",
       "│ ---     ┆ ---     ┆ ---    ┆ --- ┆   ┆ ---     ┆ ---         ┆ ---        ┆ ---            │\n",
       "│ str     ┆ str     ┆ str    ┆ i64 ┆   ┆ f64     ┆ str         ┆ str        ┆ str            │\n",
       "╞═════════╪═════════╪════════╪═════╪═══╪═════════╪═════════════╪════════════╪════════════════╡\n",
       "│ I317333 ┆ C111565 ┆ Male   ┆ 21  ┆ … ┆ 1800.51 ┆ Debit Card  ┆ 12/12/2021 ┆ Forum Istanbul │\n",
       "│ I127801 ┆ C266599 ┆ Male   ┆ 20  ┆ … ┆ 300.08  ┆ Cash        ┆ 9/11/2021  ┆ Metrocity      │\n",
       "│ I173702 ┆ C988172 ┆ Female ┆ 66  ┆ … ┆ 3000.85 ┆ Credit Card ┆ 16/05/2021 ┆ Metropol AVM   │\n",
       "│ I337046 ┆ C189076 ┆ Female ┆ 53  ┆ … ┆ 60.6    ┆ Cash        ┆ 24/10/2021 ┆ Kanyon         │\n",
       "│ I227836 ┆ C657758 ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card ┆ 24/05/2022 ┆ Forum Istanbul │\n",
       "└─────────┴─────────┴────────┴─────┴───┴─────────┴─────────────┴────────────┴────────────────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_csv('../data/customer_shopping_data_no_header.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>column_1</th><th>column_2</th><th>column_3</th><th>column_4</th><th>column_5</th><th>column_6</th><th>column_7</th><th>column_8</th><th>column_9</th><th>column_10</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;I138884&quot;</td><td>&quot;C241288&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>&quot;5/8/2022&quot;</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>&quot;12/12/2021&quot;</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>&quot;9/11/2021&quot;</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>&quot;16/05/2021&quot;</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>&quot;24/10/2021&quot;</td><td>&quot;Kanyon&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌──────────┬──────────┬──────────┬──────────┬───┬──────────┬─────────────┬────────────┬────────────┐\n",
       "│ column_1 ┆ column_2 ┆ column_3 ┆ column_4 ┆ … ┆ column_7 ┆ column_8    ┆ column_9   ┆ column_10  │\n",
       "│ ---      ┆ ---      ┆ ---      ┆ ---      ┆   ┆ ---      ┆ ---         ┆ ---        ┆ ---        │\n",
       "│ str      ┆ str      ┆ str      ┆ i64      ┆   ┆ f64      ┆ str         ┆ str        ┆ str        │\n",
       "╞══════════╪══════════╪══════════╪══════════╪═══╪══════════╪═════════════╪════════════╪════════════╡\n",
       "│ I138884  ┆ C241288  ┆ Female   ┆ 28       ┆ … ┆ 1500.4   ┆ Credit Card ┆ 5/8/2022   ┆ Kanyon     │\n",
       "│ I317333  ┆ C111565  ┆ Male     ┆ 21       ┆ … ┆ 1800.51  ┆ Debit Card  ┆ 12/12/2021 ┆ Forum      │\n",
       "│          ┆          ┆          ┆          ┆   ┆          ┆             ┆            ┆ Istanbul   │\n",
       "│ I127801  ┆ C266599  ┆ Male     ┆ 20       ┆ … ┆ 300.08   ┆ Cash        ┆ 9/11/2021  ┆ Metrocity  │\n",
       "│ I173702  ┆ C988172  ┆ Female   ┆ 66       ┆ … ┆ 3000.85  ┆ Credit Card ┆ 16/05/2021 ┆ Metropol   │\n",
       "│          ┆          ┆          ┆          ┆   ┆          ┆             ┆            ┆ AVM        │\n",
       "│ I337046  ┆ C189076  ┆ Female   ┆ 53       ┆ … ┆ 60.6     ┆ Cash        ┆ 24/10/2021 ┆ Kanyon     │\n",
       "└──────────┴──────────┴──────────┴──────────┴───┴──────────┴─────────────┴────────────┴────────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_csv('../data/customer_shopping_data_no_header.csv', has_header=False)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>invoice_no</th><th>customer_id</th><th>gender</th><th>age</th><th>category</th><th>quantity</th><th>price</th><th>payment_method</th><th>invoice_date</th><th>shopping_mall</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;I138884&quot;</td><td>&quot;C241288&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>&quot;5/8/2022&quot;</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>&quot;12/12/2021&quot;</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>&quot;9/11/2021&quot;</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>&quot;16/05/2021&quot;</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>&quot;24/10/2021&quot;</td><td>&quot;Kanyon&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌────────────┬─────────────┬────────┬─────┬───┬─────────┬──────────────┬─────────────┬─────────────┐\n",
       "│ invoice_no ┆ customer_id ┆ gender ┆ age ┆ … ┆ price   ┆ payment_meth ┆ invoice_dat ┆ shopping_ma │\n",
       "│ ---        ┆ ---         ┆ ---    ┆ --- ┆   ┆ ---     ┆ od           ┆ e           ┆ ll          │\n",
       "│ str        ┆ str         ┆ str    ┆ i64 ┆   ┆ f64     ┆ ---          ┆ ---         ┆ ---         │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆ str          ┆ str         ┆ str         │\n",
       "╞════════════╪═════════════╪════════╪═════╪═══╪═════════╪══════════════╪═════════════╪═════════════╡\n",
       "│ I138884    ┆ C241288     ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card  ┆ 5/8/2022    ┆ Kanyon      │\n",
       "│ I317333    ┆ C111565     ┆ Male   ┆ 21  ┆ … ┆ 1800.51 ┆ Debit Card   ┆ 12/12/2021  ┆ Forum       │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ Istanbul    │\n",
       "│ I127801    ┆ C266599     ┆ Male   ┆ 20  ┆ … ┆ 300.08  ┆ Cash         ┆ 9/11/2021   ┆ Metrocity   │\n",
       "│ I173702    ┆ C988172     ┆ Female ┆ 66  ┆ … ┆ 3000.85 ┆ Credit Card  ┆ 16/05/2021  ┆ Metropol    │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ AVM         │\n",
       "│ I337046    ┆ C189076     ┆ Female ┆ 53  ┆ … ┆ 60.6    ┆ Cash         ┆ 24/10/2021  ┆ Kanyon      │\n",
       "└────────────┴─────────────┴────────┴─────┴───┴─────────┴──────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_names = ['invoice_no', 'customer_id', 'gender', 'age', 'category', 'quantity', 'price', 'payment_method', 'invoice_date', 'shopping_mall']\n",
    "df = pl.read_csv('../data/customer_shopping_data_no_header.csv', \n",
    "                 has_header=False, \n",
    "                 new_columns=column_names)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>invoice_no</th><th>customer_id</th><th>gender</th><th>age</th><th>category</th><th>quantity</th><th>price</th><th>payment_method</th><th>invoice_date</th><th>shopping_mall</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>date</td><td>str</td></tr></thead><tbody><tr><td>&quot;I138884&quot;</td><td>&quot;C241288&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>2022-08-05</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>2021-12-12</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>2021-11-09</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>2021-05-16</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>2021-10-24</td><td>&quot;Kanyon&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌────────────┬─────────────┬────────┬─────┬───┬─────────┬──────────────┬─────────────┬─────────────┐\n",
       "│ invoice_no ┆ customer_id ┆ gender ┆ age ┆ … ┆ price   ┆ payment_meth ┆ invoice_dat ┆ shopping_ma │\n",
       "│ ---        ┆ ---         ┆ ---    ┆ --- ┆   ┆ ---     ┆ od           ┆ e           ┆ ll          │\n",
       "│ str        ┆ str         ┆ str    ┆ i64 ┆   ┆ f64     ┆ ---          ┆ ---         ┆ ---         │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆ str          ┆ date        ┆ str         │\n",
       "╞════════════╪═════════════╪════════╪═════╪═══╪═════════╪══════════════╪═════════════╪═════════════╡\n",
       "│ I138884    ┆ C241288     ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card  ┆ 2022-08-05  ┆ Kanyon      │\n",
       "│ I317333    ┆ C111565     ┆ Male   ┆ 21  ┆ … ┆ 1800.51 ┆ Debit Card   ┆ 2021-12-12  ┆ Forum       │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ Istanbul    │\n",
       "│ I127801    ┆ C266599     ┆ Male   ┆ 20  ┆ … ┆ 300.08  ┆ Cash         ┆ 2021-11-09  ┆ Metrocity   │\n",
       "│ I173702    ┆ C988172     ┆ Female ┆ 66  ┆ … ┆ 3000.85 ┆ Credit Card  ┆ 2021-05-16  ┆ Metropol    │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ AVM         │\n",
       "│ I337046    ┆ C189076     ┆ Female ┆ 53  ┆ … ┆ 60.6    ┆ Cash         ┆ 2021-10-24  ┆ Kanyon      │\n",
       "└────────────┴─────────────┴────────┴─────┴───┴─────────┴──────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_csv('../data/customer_shopping_data_no_header.csv', \n",
    "                 has_header=False, \n",
    "                 new_columns=column_names, \n",
    "                 try_parse_dates=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>invoice_no</th><th>customer_id</th><th>gender</th><th>age</th><th>category</th><th>quantity</th><th>price</th><th>payment_method</th><th>invoice_date</th><th>shopping_mall</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i8</td><td>str</td><td>i32</td><td>f64</td><td>str</td><td>date</td><td>str</td></tr></thead><tbody><tr><td>&quot;I138884&quot;</td><td>&quot;C241288&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>2022-08-05</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>2021-12-12</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>2021-11-09</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>2021-05-16</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>2021-10-24</td><td>&quot;Kanyon&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌────────────┬─────────────┬────────┬─────┬───┬─────────┬──────────────┬─────────────┬─────────────┐\n",
       "│ invoice_no ┆ customer_id ┆ gender ┆ age ┆ … ┆ price   ┆ payment_meth ┆ invoice_dat ┆ shopping_ma │\n",
       "│ ---        ┆ ---         ┆ ---    ┆ --- ┆   ┆ ---     ┆ od           ┆ e           ┆ ll          │\n",
       "│ str        ┆ str         ┆ str    ┆ i8  ┆   ┆ f64     ┆ ---          ┆ ---         ┆ ---         │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆ str          ┆ date        ┆ str         │\n",
       "╞════════════╪═════════════╪════════╪═════╪═══╪═════════╪══════════════╪═════════════╪═════════════╡\n",
       "│ I138884    ┆ C241288     ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card  ┆ 2022-08-05  ┆ Kanyon      │\n",
       "│ I317333    ┆ C111565     ┆ Male   ┆ 21  ┆ … ┆ 1800.51 ┆ Debit Card   ┆ 2021-12-12  ┆ Forum       │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ Istanbul    │\n",
       "│ I127801    ┆ C266599     ┆ Male   ┆ 20  ┆ … ┆ 300.08  ┆ Cash         ┆ 2021-11-09  ┆ Metrocity   │\n",
       "│ I173702    ┆ C988172     ┆ Female ┆ 66  ┆ … ┆ 3000.85 ┆ Credit Card  ┆ 2021-05-16  ┆ Metropol    │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ AVM         │\n",
       "│ I337046    ┆ C189076     ┆ Female ┆ 53  ┆ … ┆ 60.6    ┆ Cash         ┆ 2021-10-24  ┆ Kanyon      │\n",
       "└────────────┴─────────────┴────────┴─────┴───┴─────────┴──────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_csv('../data/customer_shopping_data_no_header.csv', \n",
    "                 has_header=False, \n",
    "                 new_columns=column_names, \n",
    "                 try_parse_dates=True, \n",
    "                 schema_overrides={'age': pl.Int8, 'quantity': pl.Int32})\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.write_csv('../data/output/shopping_data_output.csv', \n",
    "             include_header=False, \n",
    "             separator=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>invoice_no</th><th>customer_id</th><th>gender</th><th>age</th><th>category</th><th>quantity</th><th>price</th><th>payment_method</th><th>invoice_date</th><th>shopping_mall</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i8</td><td>str</td><td>i32</td><td>f64</td><td>str</td><td>date</td><td>str</td></tr></thead><tbody><tr><td>&quot;I138884&quot;</td><td>&quot;C241288&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>2022-08-05</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>2021-12-12</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>2021-11-09</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>2021-05-16</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>2021-10-24</td><td>&quot;Kanyon&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌────────────┬─────────────┬────────┬─────┬───┬─────────┬──────────────┬─────────────┬─────────────┐\n",
       "│ invoice_no ┆ customer_id ┆ gender ┆ age ┆ … ┆ price   ┆ payment_meth ┆ invoice_dat ┆ shopping_ma │\n",
       "│ ---        ┆ ---         ┆ ---    ┆ --- ┆   ┆ ---     ┆ od           ┆ e           ┆ ll          │\n",
       "│ str        ┆ str         ┆ str    ┆ i8  ┆   ┆ f64     ┆ ---          ┆ ---         ┆ ---         │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆ str          ┆ date        ┆ str         │\n",
       "╞════════════╪═════════════╪════════╪═════╪═══╪═════════╪══════════════╪═════════════╪═════════════╡\n",
       "│ I138884    ┆ C241288     ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card  ┆ 2022-08-05  ┆ Kanyon      │\n",
       "│ I317333    ┆ C111565     ┆ Male   ┆ 21  ┆ … ┆ 1800.51 ┆ Debit Card   ┆ 2021-12-12  ┆ Forum       │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ Istanbul    │\n",
       "│ I127801    ┆ C266599     ┆ Male   ┆ 20  ┆ … ┆ 300.08  ┆ Cash         ┆ 2021-11-09  ┆ Metrocity   │\n",
       "│ I173702    ┆ C988172     ┆ Female ┆ 66  ┆ … ┆ 3000.85 ┆ Credit Card  ┆ 2021-05-16  ┆ Metropol    │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ AVM         │\n",
       "│ I337046    ┆ C189076     ┆ Female ┆ 53  ┆ … ┆ 60.6    ┆ Cash         ┆ 2021-10-24  ┆ Kanyon      │\n",
       "└────────────┴─────────────┴────────┴─────┴───┴─────────┴──────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lf = pl.scan_csv('../data/customer_shopping_data_no_header.csv', \n",
    "                 has_header=False, \n",
    "                 new_columns=column_names, \n",
    "                 try_parse_dates=True, \n",
    "                 schema_overrides={'age': pl.Int8, 'quantity': pl.Int32})\n",
    "lf.head().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf.sink_csv('../data/output/shopping_data_output_sink.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and writing parquet files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Company</th><th>Amount</th><th>Valuation</th><th>Industry</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;OpenAI&quot;</td><td>&quot;$10,000,000,00…</td><td>&quot;n/a&quot;</td><td>&quot;Artificial int…</td></tr><tr><td>&quot;Stripe&quot;</td><td>&quot;$6,500,000,000…</td><td>&quot;$50,000,000,00…</td><td>&quot;Fintech&quot;</td></tr><tr><td>&quot;Inflection AI&quot;</td><td>&quot;$1,300,000,000…</td><td>&quot;$4,000,000,000…</td><td>&quot;Artificial int…</td></tr><tr><td>&quot;Anthropic&quot;</td><td>&quot;$1,250,000,000…</td><td>&quot;$4,000,000,000…</td><td>&quot;Artificial int…</td></tr><tr><td>&quot;Generate Capit…</td><td>&quot;$1,030,900,000…</td><td>&quot;n/a&quot;</td><td>&quot;Energy&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌──────────────────┬─────────────────┬─────────────────┬─────────────────────────┐\n",
       "│ Company          ┆ Amount          ┆ Valuation       ┆ Industry                │\n",
       "│ ---              ┆ ---             ┆ ---             ┆ ---                     │\n",
       "│ str              ┆ str             ┆ str             ┆ str                     │\n",
       "╞══════════════════╪═════════════════╪═════════════════╪═════════════════════════╡\n",
       "│ OpenAI           ┆ $10,000,000,000 ┆ n/a             ┆ Artificial intelligence │\n",
       "│ Stripe           ┆ $6,500,000,000  ┆ $50,000,000,000 ┆ Fintech                 │\n",
       "│ Inflection AI    ┆ $1,300,000,000  ┆ $4,000,000,000  ┆ Artificial intelligence │\n",
       "│ Anthropic        ┆ $1,250,000,000  ┆ $4,000,000,000  ┆ Artificial intelligence │\n",
       "│ Generate Capital ┆ $1,030,900,000  ┆ n/a             ┆ Energy                  │\n",
       "└──────────────────┴─────────────────┴─────────────────┴─────────────────────────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parquet_input_file_path = '../data/venture_funding_deals.parquet'\n",
    "df = pl.read_parquet(parquet_input_file_path, \n",
    "                     columns=['Company', 'Amount', 'Valuation', 'Industry'], \n",
    "                     row_index_name='row_cnt')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Company': String,\n",
       " 'Amount': String,\n",
       " 'Lead investors': String,\n",
       " 'Valuation': String,\n",
       " 'Industry': String,\n",
       " 'Date reported': String}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.read_parquet_schema(parquet_input_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "parquet_output_file_path = '../data/output/venture_funding_deals_output.parquet'\n",
    "df.write_parquet(parquet_output_file_path, compression='lz4', compression_level=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Company</th><th>Amount</th><th>Lead investors</th><th>Valuation</th><th>Industry</th><th>Date reported</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;OpenAI&quot;</td><td>&quot;$10,000,000,00…</td><td>&quot;Microsoft&quot;</td><td>&quot;n/a&quot;</td><td>&quot;Artificial int…</td><td>&quot;1/23/23&quot;</td></tr><tr><td>&quot;Stripe&quot;</td><td>&quot;$6,500,000,000…</td><td>&quot;n/a&quot;</td><td>&quot;$50,000,000,00…</td><td>&quot;Fintech&quot;</td><td>&quot;3/15/23&quot;</td></tr><tr><td>&quot;Inflection AI&quot;</td><td>&quot;$1,300,000,000…</td><td>&quot;Microsoft, Rei…</td><td>&quot;$4,000,000,000…</td><td>&quot;Artificial int…</td><td>&quot;6/29/23&quot;</td></tr><tr><td>&quot;Anthropic&quot;</td><td>&quot;$1,250,000,000…</td><td>&quot;Amazon&quot;</td><td>&quot;$4,000,000,000…</td><td>&quot;Artificial int…</td><td>&quot;9/25/23&quot;</td></tr><tr><td>&quot;Generate Capit…</td><td>&quot;$1,030,900,000…</td><td>&quot;n/a&quot;</td><td>&quot;n/a&quot;</td><td>&quot;Energy&quot;</td><td>&quot;1/6/23&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌────────────────┬────────────────┬────────────────┬───────────────┬───────────────┬───────────────┐\n",
       "│ Company        ┆ Amount         ┆ Lead investors ┆ Valuation     ┆ Industry      ┆ Date reported │\n",
       "│ ---            ┆ ---            ┆ ---            ┆ ---           ┆ ---           ┆ ---           │\n",
       "│ str            ┆ str            ┆ str            ┆ str           ┆ str           ┆ str           │\n",
       "╞════════════════╪════════════════╪════════════════╪═══════════════╪═══════════════╪═══════════════╡\n",
       "│ OpenAI         ┆ $10,000,000,00 ┆ Microsoft      ┆ n/a           ┆ Artificial    ┆ 1/23/23       │\n",
       "│                ┆ 0              ┆                ┆               ┆ intelligence  ┆               │\n",
       "│ Stripe         ┆ $6,500,000,000 ┆ n/a            ┆ $50,000,000,0 ┆ Fintech       ┆ 3/15/23       │\n",
       "│                ┆                ┆                ┆ 00            ┆               ┆               │\n",
       "│ Inflection AI  ┆ $1,300,000,000 ┆ Microsoft,     ┆ $4,000,000,00 ┆ Artificial    ┆ 6/29/23       │\n",
       "│                ┆                ┆ Reid Hoffman,  ┆ 0             ┆ intelligence  ┆               │\n",
       "│                ┆                ┆ Bill Ga…       ┆               ┆               ┆               │\n",
       "│ Anthropic      ┆ $1,250,000,000 ┆ Amazon         ┆ $4,000,000,00 ┆ Artificial    ┆ 9/25/23       │\n",
       "│                ┆                ┆                ┆ 0             ┆ intelligence  ┆               │\n",
       "│ Generate       ┆ $1,030,900,000 ┆ n/a            ┆ n/a           ┆ Energy        ┆ 1/6/23        │\n",
       "│ Capital        ┆                ┆                ┆               ┆               ┆               │\n",
       "└────────────────┴────────────────┴────────────────┴───────────────┴───────────────┴───────────────┘"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lf = pl.scan_parquet(parquet_input_file_path)\n",
    "lf.head().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf.sink_parquet(parquet_output_file_path, maintain_order=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Company</th><th>Amount</th><th>Lead investors</th><th>Valuation</th><th>Date reported</th><th>Industry</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>cat</td></tr></thead><tbody><tr><td>&quot;Restaurant365&quot;</td><td>&quot;$135,000,000&quot;</td><td>&quot;KKR, L Cattert…</td><td>&quot;$1,000,000,000…</td><td>&quot;5/19/23&quot;</td><td>&quot;Accounting&quot;</td></tr><tr><td>&quot;Madhive&quot;</td><td>&quot;$300,000,000&quot;</td><td>&quot;Goldman Sachs …</td><td>&quot;$1,000,000,000…</td><td>&quot;6/13/23&quot;</td><td>&quot;Advertising&quot;</td></tr><tr><td>&quot;Ursa Major,&quot;</td><td>&quot;$100,000,000&quot;</td><td>&quot;BlackRock, Spa…</td><td>&quot;n/a&quot;</td><td>&quot;4/26/23&quot;</td><td>&quot;Aerospace&quot;</td></tr><tr><td>&quot;Indigo&quot;</td><td>&quot;$250,000,000&quot;</td><td>&quot;Flagship Pione…</td><td>&quot;na&quot;</td><td>&quot;9/15/23&quot;</td><td>&quot;Agriculture&quot;</td></tr><tr><td>&quot;Chronosphere&quot;</td><td>&quot;$115,000,000&quot;</td><td>&quot;GV&quot;</td><td>&quot;n/a&quot;</td><td>&quot;1/9/23&quot;</td><td>&quot;Analytics&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌───────────────┬──────────────┬────────────────────┬────────────────┬───────────────┬─────────────┐\n",
       "│ Company       ┆ Amount       ┆ Lead investors     ┆ Valuation      ┆ Date reported ┆ Industry    │\n",
       "│ ---           ┆ ---          ┆ ---                ┆ ---            ┆ ---           ┆ ---         │\n",
       "│ str           ┆ str          ┆ str                ┆ str            ┆ str           ┆ cat         │\n",
       "╞═══════════════╪══════════════╪════════════════════╪════════════════╪═══════════════╪═════════════╡\n",
       "│ Restaurant365 ┆ $135,000,000 ┆ KKR, L Catterton   ┆ $1,000,000,000 ┆ 5/19/23       ┆ Accounting  │\n",
       "│ Madhive       ┆ $300,000,000 ┆ Goldman Sachs      ┆ $1,000,000,000 ┆ 6/13/23       ┆ Advertising │\n",
       "│               ┆              ┆ Asset Management   ┆                ┆               ┆             │\n",
       "│ Ursa Major,   ┆ $100,000,000 ┆ BlackRock, Space   ┆ n/a            ┆ 4/26/23       ┆ Aerospace   │\n",
       "│               ┆              ┆ Capital            ┆                ┆               ┆             │\n",
       "│ Indigo        ┆ $250,000,000 ┆ Flagship           ┆ na             ┆ 9/15/23       ┆ Agriculture │\n",
       "│               ┆              ┆ Pioneering, State  ┆                ┆               ┆             │\n",
       "│               ┆              ┆ of Mi…             ┆                ┆               ┆             │\n",
       "│ Chronosphere  ┆ $115,000,000 ┆ GV                 ┆ n/a            ┆ 1/9/23        ┆ Analytics   │\n",
       "└───────────────┴──────────────┴────────────────────┴────────────────┴───────────────┴─────────────┘"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partitioned_parquet_input_file_path = '../data/venture_funding_deals_partitioned'\n",
    "df = pl.read_parquet(\n",
    "    partitioned_parquet_input_file_path, \n",
    "    use_pyarrow=True, \n",
    "    pyarrow_options={'partitioning': 'hive'}\n",
    ")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "partitioned_parquet_output_file_path = '../data/output/venture_funding_deals_partitioned_output'\n",
    "df.write_parquet(\n",
    "    partitioned_parquet_output_file_path, \n",
    "    use_pyarrow=True, \n",
    "    pyarrow_options={\n",
    "        'partition_cols': ['Industry'],\n",
    "        'existing_data_behavior': 'overwrite_or_ignore'\n",
    "        }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and writing Delta Lake tables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Company</th><th>Amount</th><th>Lead investors</th><th>Valuation</th><th>Industry</th><th>Date reported</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;OpenAI&quot;</td><td>&quot;$10,000,000,00…</td><td>&quot;Microsoft&quot;</td><td>&quot;n/a&quot;</td><td>&quot;Artificial int…</td><td>&quot;1/23/23&quot;</td></tr><tr><td>&quot;Stripe&quot;</td><td>&quot;$6,500,000,000…</td><td>&quot;n/a&quot;</td><td>&quot;$50,000,000,00…</td><td>&quot;Fintech&quot;</td><td>&quot;3/15/23&quot;</td></tr><tr><td>&quot;Inflection AI&quot;</td><td>&quot;$1,300,000,000…</td><td>&quot;Microsoft, Rei…</td><td>&quot;$4,000,000,000…</td><td>&quot;Artificial int…</td><td>&quot;6/29/23&quot;</td></tr><tr><td>&quot;Anthropic&quot;</td><td>&quot;$1,250,000,000…</td><td>&quot;Amazon&quot;</td><td>&quot;$4,000,000,000…</td><td>&quot;Artificial int…</td><td>&quot;9/25/23&quot;</td></tr><tr><td>&quot;Generate Capit…</td><td>&quot;$1,030,900,000…</td><td>&quot;n/a&quot;</td><td>&quot;n/a&quot;</td><td>&quot;Energy&quot;</td><td>&quot;1/6/23&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌────────────────┬────────────────┬────────────────┬───────────────┬───────────────┬───────────────┐\n",
       "│ Company        ┆ Amount         ┆ Lead investors ┆ Valuation     ┆ Industry      ┆ Date reported │\n",
       "│ ---            ┆ ---            ┆ ---            ┆ ---           ┆ ---           ┆ ---           │\n",
       "│ str            ┆ str            ┆ str            ┆ str           ┆ str           ┆ str           │\n",
       "╞════════════════╪════════════════╪════════════════╪═══════════════╪═══════════════╪═══════════════╡\n",
       "│ OpenAI         ┆ $10,000,000,00 ┆ Microsoft      ┆ n/a           ┆ Artificial    ┆ 1/23/23       │\n",
       "│                ┆ 0              ┆                ┆               ┆ intelligence  ┆               │\n",
       "│ Stripe         ┆ $6,500,000,000 ┆ n/a            ┆ $50,000,000,0 ┆ Fintech       ┆ 3/15/23       │\n",
       "│                ┆                ┆                ┆ 00            ┆               ┆               │\n",
       "│ Inflection AI  ┆ $1,300,000,000 ┆ Microsoft,     ┆ $4,000,000,00 ┆ Artificial    ┆ 6/29/23       │\n",
       "│                ┆                ┆ Reid Hoffman,  ┆ 0             ┆ intelligence  ┆               │\n",
       "│                ┆                ┆ Bill Ga…       ┆               ┆               ┆               │\n",
       "│ Anthropic      ┆ $1,250,000,000 ┆ Amazon         ┆ $4,000,000,00 ┆ Artificial    ┆ 9/25/23       │\n",
       "│                ┆                ┆                ┆ 0             ┆ intelligence  ┆               │\n",
       "│ Generate       ┆ $1,030,900,000 ┆ n/a            ┆ n/a           ┆ Energy        ┆ 1/6/23        │\n",
       "│ Capital        ┆                ┆                ┆               ┆               ┆               │\n",
       "└────────────────┴────────────────┴────────────────┴───────────────┴───────────────┴───────────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delta_input_file_path = '../data/venture_funding_deals_delta'\n",
    "df = pl.read_delta(delta_input_file_path)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Company</th><th>Amount</th><th>Lead investors</th><th>Valuation</th><th>Industry</th><th>Date reported</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;OpenAI&quot;</td><td>&quot;$10,000,000,00…</td><td>&quot;Microsoft&quot;</td><td>&quot;n/a&quot;</td><td>&quot;Artificial int…</td><td>&quot;1/23/23&quot;</td></tr><tr><td>&quot;Stripe&quot;</td><td>&quot;$6,500,000,000…</td><td>&quot;n/a&quot;</td><td>&quot;$50,000,000,00…</td><td>&quot;Fintech&quot;</td><td>&quot;3/15/23&quot;</td></tr><tr><td>&quot;Inflection AI&quot;</td><td>&quot;$1,300,000,000…</td><td>&quot;Microsoft, Rei…</td><td>&quot;$4,000,000,000…</td><td>&quot;Artificial int…</td><td>&quot;6/29/23&quot;</td></tr><tr><td>&quot;Anthropic&quot;</td><td>&quot;$1,250,000,000…</td><td>&quot;Amazon&quot;</td><td>&quot;$4,000,000,000…</td><td>&quot;Artificial int…</td><td>&quot;9/25/23&quot;</td></tr><tr><td>&quot;Generate Capit…</td><td>&quot;$1,030,900,000…</td><td>&quot;n/a&quot;</td><td>&quot;n/a&quot;</td><td>&quot;Energy&quot;</td><td>&quot;1/6/23&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌────────────────┬────────────────┬────────────────┬───────────────┬───────────────┬───────────────┐\n",
       "│ Company        ┆ Amount         ┆ Lead investors ┆ Valuation     ┆ Industry      ┆ Date reported │\n",
       "│ ---            ┆ ---            ┆ ---            ┆ ---           ┆ ---           ┆ ---           │\n",
       "│ str            ┆ str            ┆ str            ┆ str           ┆ str           ┆ str           │\n",
       "╞════════════════╪════════════════╪════════════════╪═══════════════╪═══════════════╪═══════════════╡\n",
       "│ OpenAI         ┆ $10,000,000,00 ┆ Microsoft      ┆ n/a           ┆ Artificial    ┆ 1/23/23       │\n",
       "│                ┆ 0              ┆                ┆               ┆ intelligence  ┆               │\n",
       "│ Stripe         ┆ $6,500,000,000 ┆ n/a            ┆ $50,000,000,0 ┆ Fintech       ┆ 3/15/23       │\n",
       "│                ┆                ┆                ┆ 00            ┆               ┆               │\n",
       "│ Inflection AI  ┆ $1,300,000,000 ┆ Microsoft,     ┆ $4,000,000,00 ┆ Artificial    ┆ 6/29/23       │\n",
       "│                ┆                ┆ Reid Hoffman,  ┆ 0             ┆ intelligence  ┆               │\n",
       "│                ┆                ┆ Bill Ga…       ┆               ┆               ┆               │\n",
       "│ Anthropic      ┆ $1,250,000,000 ┆ Amazon         ┆ $4,000,000,00 ┆ Artificial    ┆ 9/25/23       │\n",
       "│                ┆                ┆                ┆ 0             ┆ intelligence  ┆               │\n",
       "│ Generate       ┆ $1,030,900,000 ┆ n/a            ┆ n/a           ┆ Energy        ┆ 1/6/23        │\n",
       "│ Capital        ┆                ┆                ┆               ┆               ┆               │\n",
       "└────────────────┴────────────────┴────────────────┴───────────────┴───────────────┴───────────────┘"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lf = pl.scan_delta(delta_input_file_path)\n",
    "lf.head().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.write_delta('../data/output/venture_funding_deals_delta_output', mode='overwrite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta_partitioned_output_file_path = '../data/output/venture_funding_deals_delta_partitioned_output'\n",
    "delta_write_options = {'partition_by': 'Industry'}\n",
    "df.write_delta(\n",
    "    delta_partitioned_output_file_path, \n",
    "    mode='overwrite', \n",
    "    delta_write_options=delta_write_options\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Company</th><th>Amount</th><th>Lead investors</th><th>Valuation</th><th>Industry</th><th>Date reported</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;Stripe&quot;</td><td>&quot;$6,500,000,000…</td><td>&quot;n/a&quot;</td><td>&quot;$50,000,000,00…</td><td>&quot;Fintech&quot;</td><td>&quot;3/15/23&quot;</td></tr><tr><td>&quot;Ramp&quot;</td><td>&quot;$300,000,000&quot;</td><td>&quot;n/a&quot;</td><td>&quot;$5,800,000,000…</td><td>&quot;Fintech&quot;</td><td>&quot;8/23/23&quot;</td></tr><tr><td>&quot;Clear Street&quot;</td><td>&quot;$270,000,000&quot;</td><td>&quot;Prysm Capital&quot;</td><td>&quot;$2,000,000,000…</td><td>&quot;Fintech&quot;</td><td>&quot;4/11/23&quot;</td></tr><tr><td>&quot;PayJoy&quot;</td><td>&quot;$150,000,000&quot;</td><td>&quot;Warburg Pincus…</td><td>&quot;n/a&quot;</td><td>&quot;Fintech&quot;</td><td>&quot;9/5/23&quot;</td></tr><tr><td>&quot;Teamshares&quot;</td><td>&quot;$124,000,000&quot;</td><td>&quot;QED Investors&quot;</td><td>&quot;n/a&quot;</td><td>&quot;Fintech&quot;</td><td>&quot;8/18/23&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌──────────────┬────────────────┬────────────────┬─────────────────┬──────────┬───────────────┐\n",
       "│ Company      ┆ Amount         ┆ Lead investors ┆ Valuation       ┆ Industry ┆ Date reported │\n",
       "│ ---          ┆ ---            ┆ ---            ┆ ---             ┆ ---      ┆ ---           │\n",
       "│ str          ┆ str            ┆ str            ┆ str             ┆ str      ┆ str           │\n",
       "╞══════════════╪════════════════╪════════════════╪═════════════════╪══════════╪═══════════════╡\n",
       "│ Stripe       ┆ $6,500,000,000 ┆ n/a            ┆ $50,000,000,000 ┆ Fintech  ┆ 3/15/23       │\n",
       "│ Ramp         ┆ $300,000,000   ┆ n/a            ┆ $5,800,000,000  ┆ Fintech  ┆ 8/23/23       │\n",
       "│ Clear Street ┆ $270,000,000   ┆ Prysm Capital  ┆ $2,000,000,000  ┆ Fintech  ┆ 4/11/23       │\n",
       "│ PayJoy       ┆ $150,000,000   ┆ Warburg Pincus ┆ n/a             ┆ Fintech  ┆ 9/5/23        │\n",
       "│ Teamshares   ┆ $124,000,000   ┆ QED Investors  ┆ n/a             ┆ Fintech  ┆ 8/18/23       │\n",
       "└──────────────┴────────────────┴────────────────┴─────────────────┴──────────┴───────────────┘"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_delta(delta_partitioned_output_file_path)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Company</th><th>Amount</th><th>Lead investors</th><th>Valuation</th><th>Industry</th><th>Date reported</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;Restaurant365&quot;</td><td>&quot;$135,000,000&quot;</td><td>&quot;KKR, L Cattert…</td><td>&quot;$1,000,000,000…</td><td>&quot;Accounting&quot;</td><td>&quot;5/19/23&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 6)\n",
       "┌───────────────┬──────────────┬──────────────────┬────────────────┬────────────┬───────────────┐\n",
       "│ Company       ┆ Amount       ┆ Lead investors   ┆ Valuation      ┆ Industry   ┆ Date reported │\n",
       "│ ---           ┆ ---          ┆ ---              ┆ ---            ┆ ---        ┆ ---           │\n",
       "│ str           ┆ str          ┆ str              ┆ str            ┆ str        ┆ str           │\n",
       "╞═══════════════╪══════════════╪══════════════════╪════════════════╪════════════╪═══════════════╡\n",
       "│ Restaurant365 ┆ $135,000,000 ┆ KKR, L Catterton ┆ $1,000,000,000 ┆ Accounting ┆ 5/19/23       │\n",
       "└───────────────┴──────────────┴──────────────────┴────────────────┴────────────┴───────────────┘"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_delta(\n",
    "    delta_partitioned_output_file_path, \n",
    "    pyarrow_options={'partitions': [('Industry', '=', 'Accounting')]}\n",
    ")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from config import aws_access_key_id, aws_secret_access_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Letters</th></tr><tr><td>str</td></tr></thead><tbody><tr><td>&quot;a&quot;</td></tr><tr><td>&quot;b&quot;</td></tr><tr><td>&quot;c&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 1)\n",
       "┌─────────┐\n",
       "│ Letters │\n",
       "│ ---     │\n",
       "│ str     │\n",
       "╞═════════╡\n",
       "│ a       │\n",
       "│ b       │\n",
       "│ c       │\n",
       "└─────────┘"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_path = 's3://sandbox-data-lake/letters_delta'\n",
    "storage_options= {\n",
    "    'aws_access_key_id': aws_access_key_id,\n",
    "    'aws_secret_access_key': aws_secret_access_key,\n",
    "    'aws_region': 'us-west-1'\n",
    "}\n",
    "\n",
    "df = pl.read_delta(table_path, storage_options=storage_options)  \n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and writing JSON files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>place</th><th>pop1980</th><th>pop2000</th><th>pop2010</th><th>pop2022</th><th>pop2023</th><th>pop2030</th><th>pop2050</th><th>country</th><th>area</th></tr><tr><td>i64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>i64</td><td>f64</td><td>f64</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>356</td><td>6.96828385e8</td><td>1.0596e9</td><td>1.2406e9</td><td>1.4172e9</td><td>1428627663</td><td>1.5150e9</td><td>1.6705e9</td><td>&quot;India&quot;</td><td>3287590</td></tr><tr><td>156</td><td>9.82372466e8</td><td>1.2641e9</td><td>1.3482e9</td><td>1.4259e9</td><td>1425671352</td><td>1.4156e9</td><td>1.3126e9</td><td>&quot;China&quot;</td><td>9706961</td></tr><tr><td>840</td><td>2.23140018e8</td><td>2.82398554e8</td><td>3.11182845e8</td><td>3.38289857e8</td><td>339996563</td><td>3.52162301e8</td><td>3.75391963e8</td><td>&quot;United States&quot;</td><td>9372610</td></tr><tr><td>360</td><td>1.48177096e8</td><td>2.14072421e8</td><td>2.44016173e8</td><td>2.75501339e8</td><td>277534122</td><td>2.921501e8</td><td>3.17225213e8</td><td>&quot;Indonesia&quot;</td><td>1904569</td></tr><tr><td>586</td><td>8.0624057e7</td><td>1.54369924e8</td><td>1.94454498e8</td><td>2.35824862e8</td><td>240485658</td><td>2.74029836e8</td><td>3.67808468e8</td><td>&quot;Pakistan&quot;</td><td>881912</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌───────┬────────────┬────────────┬────────────┬───┬────────────┬────────────┬───────────┬─────────┐\n",
       "│ place ┆ pop1980    ┆ pop2000    ┆ pop2010    ┆ … ┆ pop2030    ┆ pop2050    ┆ country   ┆ area    │\n",
       "│ ---   ┆ ---        ┆ ---        ┆ ---        ┆   ┆ ---        ┆ ---        ┆ ---       ┆ ---     │\n",
       "│ i64   ┆ f64        ┆ f64        ┆ f64        ┆   ┆ f64        ┆ f64        ┆ str       ┆ i64     │\n",
       "╞═══════╪════════════╪════════════╪════════════╪═══╪════════════╪════════════╪═══════════╪═════════╡\n",
       "│ 356   ┆ 6.96828385 ┆ 1.0596e9   ┆ 1.2406e9   ┆ … ┆ 1.5150e9   ┆ 1.6705e9   ┆ India     ┆ 3287590 │\n",
       "│       ┆ e8         ┆            ┆            ┆   ┆            ┆            ┆           ┆         │\n",
       "│ 156   ┆ 9.82372466 ┆ 1.2641e9   ┆ 1.3482e9   ┆ … ┆ 1.4156e9   ┆ 1.3126e9   ┆ China     ┆ 9706961 │\n",
       "│       ┆ e8         ┆            ┆            ┆   ┆            ┆            ┆           ┆         │\n",
       "│ 840   ┆ 2.23140018 ┆ 2.82398554 ┆ 3.11182845 ┆ … ┆ 3.52162301 ┆ 3.75391963 ┆ United    ┆ 9372610 │\n",
       "│       ┆ e8         ┆ e8         ┆ e8         ┆   ┆ e8         ┆ e8         ┆ States    ┆         │\n",
       "│ 360   ┆ 1.48177096 ┆ 2.14072421 ┆ 2.44016173 ┆ … ┆ 2.921501e8 ┆ 3.17225213 ┆ Indonesia ┆ 1904569 │\n",
       "│       ┆ e8         ┆ e8         ┆ e8         ┆   ┆            ┆ e8         ┆           ┆         │\n",
       "│ 586   ┆ 8.0624057e ┆ 1.54369924 ┆ 1.94454498 ┆ … ┆ 2.74029836 ┆ 3.67808468 ┆ Pakistan  ┆ 881912  │\n",
       "│       ┆ 7          ┆ e8         ┆ e8         ┆   ┆ e8         ┆ e8         ┆           ┆         │\n",
       "└───────┴────────────┴────────────┴────────────┴───┴────────────┴────────────┴───────────┴─────────┘"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_json('../data/world_population.json')\n",
    "df.select(df.columns[:10]).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.write_json('../data/output/world_population_output.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>place</th><th>pop1980</th><th>pop2000</th><th>pop2010</th><th>pop2022</th><th>pop2023</th><th>pop2030</th><th>pop2050</th><th>country</th><th>area</th></tr><tr><td>i64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>i64</td><td>f64</td><td>f64</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>356</td><td>6.96828385e8</td><td>1.0596e9</td><td>1.2406e9</td><td>1.4172e9</td><td>1428627663</td><td>1.5150e9</td><td>1.6705e9</td><td>&quot;India&quot;</td><td>3287590</td></tr><tr><td>156</td><td>9.82372466e8</td><td>1.2641e9</td><td>1.3482e9</td><td>1.4259e9</td><td>1425671352</td><td>1.4156e9</td><td>1.3126e9</td><td>&quot;China&quot;</td><td>9706961</td></tr><tr><td>840</td><td>2.23140018e8</td><td>2.82398554e8</td><td>3.11182845e8</td><td>3.38289857e8</td><td>339996563</td><td>3.52162301e8</td><td>3.75391963e8</td><td>&quot;United States&quot;</td><td>9372610</td></tr><tr><td>360</td><td>1.48177096e8</td><td>2.14072421e8</td><td>2.44016173e8</td><td>2.75501339e8</td><td>277534122</td><td>2.921501e8</td><td>3.17225213e8</td><td>&quot;Indonesia&quot;</td><td>1904569</td></tr><tr><td>586</td><td>8.0624057e7</td><td>1.54369924e8</td><td>1.94454498e8</td><td>2.35824862e8</td><td>240485658</td><td>2.74029836e8</td><td>3.67808468e8</td><td>&quot;Pakistan&quot;</td><td>881912</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌───────┬────────────┬────────────┬────────────┬───┬────────────┬────────────┬───────────┬─────────┐\n",
       "│ place ┆ pop1980    ┆ pop2000    ┆ pop2010    ┆ … ┆ pop2030    ┆ pop2050    ┆ country   ┆ area    │\n",
       "│ ---   ┆ ---        ┆ ---        ┆ ---        ┆   ┆ ---        ┆ ---        ┆ ---       ┆ ---     │\n",
       "│ i64   ┆ f64        ┆ f64        ┆ f64        ┆   ┆ f64        ┆ f64        ┆ str       ┆ i64     │\n",
       "╞═══════╪════════════╪════════════╪════════════╪═══╪════════════╪════════════╪═══════════╪═════════╡\n",
       "│ 356   ┆ 6.96828385 ┆ 1.0596e9   ┆ 1.2406e9   ┆ … ┆ 1.5150e9   ┆ 1.6705e9   ┆ India     ┆ 3287590 │\n",
       "│       ┆ e8         ┆            ┆            ┆   ┆            ┆            ┆           ┆         │\n",
       "│ 156   ┆ 9.82372466 ┆ 1.2641e9   ┆ 1.3482e9   ┆ … ┆ 1.4156e9   ┆ 1.3126e9   ┆ China     ┆ 9706961 │\n",
       "│       ┆ e8         ┆            ┆            ┆   ┆            ┆            ┆           ┆         │\n",
       "│ 840   ┆ 2.23140018 ┆ 2.82398554 ┆ 3.11182845 ┆ … ┆ 3.52162301 ┆ 3.75391963 ┆ United    ┆ 9372610 │\n",
       "│       ┆ e8         ┆ e8         ┆ e8         ┆   ┆ e8         ┆ e8         ┆ States    ┆         │\n",
       "│ 360   ┆ 1.48177096 ┆ 2.14072421 ┆ 2.44016173 ┆ … ┆ 2.921501e8 ┆ 3.17225213 ┆ Indonesia ┆ 1904569 │\n",
       "│       ┆ e8         ┆ e8         ┆ e8         ┆   ┆            ┆ e8         ┆           ┆         │\n",
       "│ 586   ┆ 8.0624057e ┆ 1.54369924 ┆ 1.94454498 ┆ … ┆ 2.74029836 ┆ 3.67808468 ┆ Pakistan  ┆ 881912  │\n",
       "│       ┆ 7          ┆ e8         ┆ e8         ┆   ┆ e8         ┆ e8         ┆           ┆         │\n",
       "└───────┴────────────┴────────────┴────────────┴───┴────────────┴────────────┴───────────┴─────────┘"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_ndjson('../data/world_population.jsonl')\n",
    "df.select(df.columns[:10]).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.write_ndjson('../data/output/world_population_output.jsonl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>place</th><th>pop1980</th><th>pop2000</th><th>pop2010</th><th>pop2022</th><th>pop2023</th><th>pop2030</th><th>pop2050</th><th>country</th><th>area</th></tr><tr><td>i64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>i64</td><td>f64</td><td>f64</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>356</td><td>6.96828385e8</td><td>1.0596e9</td><td>1.2406e9</td><td>1.4172e9</td><td>1428627663</td><td>1.5150e9</td><td>1.6705e9</td><td>&quot;India&quot;</td><td>3287590</td></tr><tr><td>156</td><td>9.82372466e8</td><td>1.2641e9</td><td>1.3482e9</td><td>1.4259e9</td><td>1425671352</td><td>1.4156e9</td><td>1.3126e9</td><td>&quot;China&quot;</td><td>9706961</td></tr><tr><td>840</td><td>2.23140018e8</td><td>2.82398554e8</td><td>3.11182845e8</td><td>3.38289857e8</td><td>339996563</td><td>3.52162301e8</td><td>3.75391963e8</td><td>&quot;United States&quot;</td><td>9372610</td></tr><tr><td>360</td><td>1.48177096e8</td><td>2.14072421e8</td><td>2.44016173e8</td><td>2.75501339e8</td><td>277534122</td><td>2.921501e8</td><td>3.17225213e8</td><td>&quot;Indonesia&quot;</td><td>1904569</td></tr><tr><td>586</td><td>8.0624057e7</td><td>1.54369924e8</td><td>1.94454498e8</td><td>2.35824862e8</td><td>240485658</td><td>2.74029836e8</td><td>3.67808468e8</td><td>&quot;Pakistan&quot;</td><td>881912</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌───────┬────────────┬────────────┬────────────┬───┬────────────┬────────────┬───────────┬─────────┐\n",
       "│ place ┆ pop1980    ┆ pop2000    ┆ pop2010    ┆ … ┆ pop2030    ┆ pop2050    ┆ country   ┆ area    │\n",
       "│ ---   ┆ ---        ┆ ---        ┆ ---        ┆   ┆ ---        ┆ ---        ┆ ---       ┆ ---     │\n",
       "│ i64   ┆ f64        ┆ f64        ┆ f64        ┆   ┆ f64        ┆ f64        ┆ str       ┆ i64     │\n",
       "╞═══════╪════════════╪════════════╪════════════╪═══╪════════════╪════════════╪═══════════╪═════════╡\n",
       "│ 356   ┆ 6.96828385 ┆ 1.0596e9   ┆ 1.2406e9   ┆ … ┆ 1.5150e9   ┆ 1.6705e9   ┆ India     ┆ 3287590 │\n",
       "│       ┆ e8         ┆            ┆            ┆   ┆            ┆            ┆           ┆         │\n",
       "│ 156   ┆ 9.82372466 ┆ 1.2641e9   ┆ 1.3482e9   ┆ … ┆ 1.4156e9   ┆ 1.3126e9   ┆ China     ┆ 9706961 │\n",
       "│       ┆ e8         ┆            ┆            ┆   ┆            ┆            ┆           ┆         │\n",
       "│ 840   ┆ 2.23140018 ┆ 2.82398554 ┆ 3.11182845 ┆ … ┆ 3.52162301 ┆ 3.75391963 ┆ United    ┆ 9372610 │\n",
       "│       ┆ e8         ┆ e8         ┆ e8         ┆   ┆ e8         ┆ e8         ┆ States    ┆         │\n",
       "│ 360   ┆ 1.48177096 ┆ 2.14072421 ┆ 2.44016173 ┆ … ┆ 2.921501e8 ┆ 3.17225213 ┆ Indonesia ┆ 1904569 │\n",
       "│       ┆ e8         ┆ e8         ┆ e8         ┆   ┆            ┆ e8         ┆           ┆         │\n",
       "│ 586   ┆ 8.0624057e ┆ 1.54369924 ┆ 1.94454498 ┆ … ┆ 2.74029836 ┆ 3.67808468 ┆ Pakistan  ┆ 881912  │\n",
       "│       ┆ 7          ┆ e8         ┆ e8         ┆   ┆ e8         ┆ e8         ┆           ┆         │\n",
       "└───────┴────────────┴────────────┴────────────┴───┴────────────┴────────────┴───────────┴─────────┘"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lf = pl.scan_ndjson('../data/world_population.jsonl')\n",
    "lf.select(lf.columns[:10]).head().collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and writing Excel files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xlsxwriter.workbook.Workbook at 0x108046910>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_file_path = '../data/output/customer_shopping_data.xlsx'\n",
    "\n",
    "df = pl.read_csv('../data/customer_shopping_data.csv')\n",
    "df.write_excel(\n",
    "    output_file_path,\n",
    "    worksheet='Output Sheet1',\n",
    "    header_format={'bold': True}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>invoice_no</th><th>customer_id</th><th>gender</th><th>age</th><th>category</th><th>quantity</th><th>price</th><th>payment_method</th><th>invoice_date</th><th>shopping_mall</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>date</td><td>str</td></tr></thead><tbody><tr><td>&quot;I138884&quot;</td><td>&quot;C241288&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>2022-08-05</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>2021-12-12</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>2021-11-09</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>2021-05-16</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>2021-10-24</td><td>&quot;Kanyon&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌────────────┬─────────────┬────────┬─────┬───┬─────────┬──────────────┬─────────────┬─────────────┐\n",
       "│ invoice_no ┆ customer_id ┆ gender ┆ age ┆ … ┆ price   ┆ payment_meth ┆ invoice_dat ┆ shopping_ma │\n",
       "│ ---        ┆ ---         ┆ ---    ┆ --- ┆   ┆ ---     ┆ od           ┆ e           ┆ ll          │\n",
       "│ str        ┆ str         ┆ str    ┆ i64 ┆   ┆ f64     ┆ ---          ┆ ---         ┆ ---         │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆ str          ┆ date        ┆ str         │\n",
       "╞════════════╪═════════════╪════════╪═════╪═══╪═════════╪══════════════╪═════════════╪═════════════╡\n",
       "│ I138884    ┆ C241288     ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card  ┆ 2022-08-05  ┆ Kanyon      │\n",
       "│ I317333    ┆ C111565     ┆ Male   ┆ 21  ┆ … ┆ 1800.51 ┆ Debit Card   ┆ 2021-12-12  ┆ Forum       │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ Istanbul    │\n",
       "│ I127801    ┆ C266599     ┆ Male   ┆ 20  ┆ … ┆ 300.08  ┆ Cash         ┆ 2021-11-09  ┆ Metrocity   │\n",
       "│ I173702    ┆ C988172     ┆ Female ┆ 66  ┆ … ┆ 3000.85 ┆ Credit Card  ┆ 2021-05-16  ┆ Metropol    │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ AVM         │\n",
       "│ I337046    ┆ C189076     ┆ Female ┆ 53  ┆ … ┆ 60.6    ┆ Cash         ┆ 2021-10-24  ┆ Kanyon      │\n",
       "└────────────┴─────────────┴────────┴─────┴───┴─────────┴──────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_file_path = '../data/output/customer_shopping_data.xlsx'\n",
    "df = pl.read_excel(\n",
    "    input_file_path, \n",
    "    sheet_name='Output Sheet1',\n",
    "    engine='xlsx2csv',\n",
    "    read_options={'try_parse_dates': True}\n",
    ")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and writing other data file formats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_input_file_path = '../data/customer_shopping_data.csv'\n",
    "ipc_file_path = '../data/output/customer_shopping_data.arrow'\n",
    "df = pl.read_csv(csv_input_file_path)\n",
    "df.write_ipc(ipc_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>invoice_no</th><th>customer_id</th><th>gender</th><th>age</th><th>category</th><th>quantity</th><th>price</th><th>payment_method</th><th>invoice_date</th><th>shopping_mall</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;I138884&quot;</td><td>&quot;C241288&quot;</td><td>&quot;Female&quot;</td><td>28</td><td>&quot;Clothing&quot;</td><td>5</td><td>1500.4</td><td>&quot;Credit Card&quot;</td><td>&quot;5/8/2022&quot;</td><td>&quot;Kanyon&quot;</td></tr><tr><td>&quot;I317333&quot;</td><td>&quot;C111565&quot;</td><td>&quot;Male&quot;</td><td>21</td><td>&quot;Shoes&quot;</td><td>3</td><td>1800.51</td><td>&quot;Debit Card&quot;</td><td>&quot;12/12/2021&quot;</td><td>&quot;Forum Istanbul…</td></tr><tr><td>&quot;I127801&quot;</td><td>&quot;C266599&quot;</td><td>&quot;Male&quot;</td><td>20</td><td>&quot;Clothing&quot;</td><td>1</td><td>300.08</td><td>&quot;Cash&quot;</td><td>&quot;9/11/2021&quot;</td><td>&quot;Metrocity&quot;</td></tr><tr><td>&quot;I173702&quot;</td><td>&quot;C988172&quot;</td><td>&quot;Female&quot;</td><td>66</td><td>&quot;Shoes&quot;</td><td>5</td><td>3000.85</td><td>&quot;Credit Card&quot;</td><td>&quot;16/05/2021&quot;</td><td>&quot;Metropol AVM&quot;</td></tr><tr><td>&quot;I337046&quot;</td><td>&quot;C189076&quot;</td><td>&quot;Female&quot;</td><td>53</td><td>&quot;Books&quot;</td><td>4</td><td>60.6</td><td>&quot;Cash&quot;</td><td>&quot;24/10/2021&quot;</td><td>&quot;Kanyon&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌────────────┬─────────────┬────────┬─────┬───┬─────────┬──────────────┬─────────────┬─────────────┐\n",
       "│ invoice_no ┆ customer_id ┆ gender ┆ age ┆ … ┆ price   ┆ payment_meth ┆ invoice_dat ┆ shopping_ma │\n",
       "│ ---        ┆ ---         ┆ ---    ┆ --- ┆   ┆ ---     ┆ od           ┆ e           ┆ ll          │\n",
       "│ str        ┆ str         ┆ str    ┆ i64 ┆   ┆ f64     ┆ ---          ┆ ---         ┆ ---         │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆ str          ┆ str         ┆ str         │\n",
       "╞════════════╪═════════════╪════════╪═════╪═══╪═════════╪══════════════╪═════════════╪═════════════╡\n",
       "│ I138884    ┆ C241288     ┆ Female ┆ 28  ┆ … ┆ 1500.4  ┆ Credit Card  ┆ 5/8/2022    ┆ Kanyon      │\n",
       "│ I317333    ┆ C111565     ┆ Male   ┆ 21  ┆ … ┆ 1800.51 ┆ Debit Card   ┆ 12/12/2021  ┆ Forum       │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ Istanbul    │\n",
       "│ I127801    ┆ C266599     ┆ Male   ┆ 20  ┆ … ┆ 300.08  ┆ Cash         ┆ 9/11/2021   ┆ Metrocity   │\n",
       "│ I173702    ┆ C988172     ┆ Female ┆ 66  ┆ … ┆ 3000.85 ┆ Credit Card  ┆ 16/05/2021  ┆ Metropol    │\n",
       "│            ┆             ┆        ┆     ┆   ┆         ┆              ┆             ┆ AVM         │\n",
       "│ I337046    ┆ C189076     ┆ Female ┆ 53  ┆ … ┆ 60.6    ┆ Cash         ┆ 24/10/2021  ┆ Kanyon      │\n",
       "└────────────┴─────────────┴────────┴─────┴───┴─────────┴──────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_ipc(ipc_file_path)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "avro_file_path = '../data/world_population.avro'\n",
    "df = (\n",
    "    pl.read_json('../data/world_population.json')\n",
    "    .select(['country', 'pop2023', 'density'])\n",
    ")\n",
    "df.write_avro(avro_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>country</th><th>pop2023</th><th>density</th></tr><tr><td>str</td><td>i64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;India&quot;</td><td>1428627663</td><td>480.5033</td></tr><tr><td>&quot;China&quot;</td><td>1425671352</td><td>151.2696</td></tr><tr><td>&quot;United States&quot;</td><td>339996563</td><td>37.1686</td></tr><tr><td>&quot;Indonesia&quot;</td><td>277534122</td><td>147.8196</td></tr><tr><td>&quot;Pakistan&quot;</td><td>240485658</td><td>311.9625</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌───────────────┬────────────┬──────────┐\n",
       "│ country       ┆ pop2023    ┆ density  │\n",
       "│ ---           ┆ ---        ┆ ---      │\n",
       "│ str           ┆ i64        ┆ f64      │\n",
       "╞═══════════════╪════════════╪══════════╡\n",
       "│ India         ┆ 1428627663 ┆ 480.5033 │\n",
       "│ China         ┆ 1425671352 ┆ 151.2696 │\n",
       "│ United States ┆ 339996563  ┆ 37.1686  │\n",
       "│ Indonesia     ┆ 277534122  ┆ 147.8196 │\n",
       "│ Pakistan      ┆ 240485658  ┆ 311.9625 │\n",
       "└───────────────┴────────────┴──────────┘"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_avro(avro_file_path)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th><th>c</th></tr><tr><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>4</td><td>7</td></tr><tr><td>2</td><td>5</td><td>8</td></tr><tr><td>3</td><td>6</td><td>9</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 3)\n",
       "┌─────┬─────┬─────┐\n",
       "│ a   ┆ b   ┆ c   │\n",
       "│ --- ┆ --- ┆ --- │\n",
       "│ i64 ┆ i64 ┆ i64 │\n",
       "╞═════╪═════╪═════╡\n",
       "│ 1   ┆ 4   ┆ 7   │\n",
       "│ 2   ┆ 5   ┆ 8   │\n",
       "│ 3   ┆ 6   ┆ 9   │\n",
       "└─────┴─────┴─────┘"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lf = pl.scan_iceberg('../data/my_iceberg_catalog/demo.db/my_table/metadata/00001-7ad1e6e8-7a0d-4455-ac6d-bcca5a45b494.metadata.json')\n",
    "lf.head().collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = pl.scan_ipc(ipc_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "InvalidOperationError",
     "evalue": "sink_Ipc(IpcWriterOptions { compression: Some(ZSTD), maintain_order: true }) not yet supported in standard engine. Use 'collect().write_parquet()'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidOperationError\u001b[0m                     Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mlf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msink_ipc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m../data/output/customer_shopping_data.arrow\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Desktop/Polars-Cookbook/.venv/lib/python3.11/site-packages/polars/utils/unstable.py:59\u001b[0m, in \u001b[0;36munstable.<locals>.decorate.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(function)\n\u001b[1;32m     57\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m     58\u001b[0m     issue_unstable_warning(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunction\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m` is considered unstable.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 59\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Desktop/Polars-Cookbook/.venv/lib/python3.11/site-packages/polars/lazyframe/frame.py:2265\u001b[0m, in \u001b[0;36mLazyFrame.sink_ipc\u001b[0;34m(self, path, compression, maintain_order, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, no_optimization)\u001b[0m\n\u001b[1;32m   2215\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   2216\u001b[0m \u001b[38;5;124;03mEvaluate the query in streaming mode and write to an IPC file.\u001b[39;00m\n\u001b[1;32m   2217\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2254\u001b[0m \u001b[38;5;124;03m>>> lf.sink_ipc(\"out.arrow\")  # doctest: +SKIP\u001b[39;00m\n\u001b[1;32m   2255\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   2256\u001b[0m lf \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_sink_optimizations(\n\u001b[1;32m   2257\u001b[0m     type_coercion\u001b[38;5;241m=\u001b[39mtype_coercion,\n\u001b[1;32m   2258\u001b[0m     predicate_pushdown\u001b[38;5;241m=\u001b[39mpredicate_pushdown,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2262\u001b[0m     no_optimization\u001b[38;5;241m=\u001b[39mno_optimization,\n\u001b[1;32m   2263\u001b[0m )\n\u001b[0;32m-> 2265\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msink_ipc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2266\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2267\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcompression\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2268\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmaintain_order\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmaintain_order\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2269\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mInvalidOperationError\u001b[0m: sink_Ipc(IpcWriterOptions { compression: Some(ZSTD), maintain_order: true }) not yet supported in standard engine. Use 'collect().write_parquet()'"
     ]
    }
   ],
   "source": [
    "lf.sink_ipc('../data/output/customer_shopping_data.arrow')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf.collect().write_ipc('../data/output/customer_shopping_data_lazy.arrow')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = pl.scan_csv(csv_input_file_path)\n",
    "lf.sink_ipc('../data/output/customer_shopping_data_from_csv.arrow')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and writing multiple files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {'Letter': ['A','B','C'], 'Value': [1,2,3]}\n",
    "df = pl.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<polars.dataframe.group_by.GroupBy object at 0x104d41450>\n"
     ]
    }
   ],
   "source": [
    "dfs = df.group_by(['Letter'])\n",
    "print(dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, df in dfs:\n",
    "    df.write_csv(f'../data/output/letter_{name[0]}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Letter</th><th>Value</th></tr><tr><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;A&quot;</td><td>1</td></tr><tr><td>&quot;B&quot;</td><td>2</td></tr><tr><td>&quot;C&quot;</td><td>3</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌────────┬───────┐\n",
       "│ Letter ┆ Value │\n",
       "│ ---    ┆ ---   │\n",
       "│ str    ┆ i64   │\n",
       "╞════════╪═══════╡\n",
       "│ A      ┆ 1     │\n",
       "│ B      ┆ 2     │\n",
       "│ C      ┆ 3     │\n",
       "└────────┴───────┘"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_csv('../data/output/letter_*.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Letter</th><th>Value</th></tr><tr><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;A&quot;</td><td>1</td></tr><tr><td>&quot;B&quot;</td><td>2</td></tr><tr><td>&quot;C&quot;</td><td>3</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌────────┬───────┐\n",
       "│ Letter ┆ Value │\n",
       "│ ---    ┆ ---   │\n",
       "│ str    ┆ i64   │\n",
       "╞════════╪═══════╡\n",
       "│ A      ┆ 1     │\n",
       "│ B      ┆ 2     │\n",
       "│ C      ┆ 3     │\n",
       "└────────┴───────┘"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lf = pl.scan_csv('../data/output/letter_*.csv')\n",
    "lf.head().collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[shape: (1, 2)\n",
       " ┌────────┬───────┐\n",
       " │ Letter ┆ Value │\n",
       " │ ---    ┆ ---   │\n",
       " │ str    ┆ i64   │\n",
       " ╞════════╪═══════╡\n",
       " │ B      ┆ 2     │\n",
       " └────────┴───────┘,\n",
       " shape: (1, 2)\n",
       " ┌────────┬───────┐\n",
       " │ Letter ┆ Value │\n",
       " │ ---    ┆ ---   │\n",
       " │ str    ┆ i64   │\n",
       " ╞════════╪═══════╡\n",
       " │ C      ┆ 3     │\n",
       " └────────┴───────┘,\n",
       " shape: (1, 2)\n",
       " ┌────────┬───────┐\n",
       " │ Letter ┆ Value │\n",
       " │ ---    ┆ ---   │\n",
       " │ str    ┆ i64   │\n",
       " ╞════════╪═══════╡\n",
       " │ A      ┆ 1     │\n",
       " └────────┴───────┘]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import glob\n",
    "lfs = [pl.scan_csv(file) for file in glob.glob('../data/output/letter_*.csv')]\n",
    "dfs = pl.collect_all(lfs)\n",
    "dfs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working with databases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from config import postgres_pass, postgres_user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>brand</th><th>model</th><th>year</th></tr><tr><td>str</td><td>str</td><td>i32</td></tr></thead><tbody><tr><td>&quot;Volvo&quot;</td><td>&quot;p1800&quot;</td><td>1968</td></tr><tr><td>&quot;BMW&quot;</td><td>&quot;M1&quot;</td><td>1978</td></tr><tr><td>&quot;Toyota&quot;</td><td>&quot;Celica&quot;</td><td>1975</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 3)\n",
       "┌────────┬────────┬──────┐\n",
       "│ brand  ┆ model  ┆ year │\n",
       "│ ---    ┆ ---    ┆ ---  │\n",
       "│ str    ┆ str    ┆ i32  │\n",
       "╞════════╪════════╪══════╡\n",
       "│ Volvo  ┆ p1800  ┆ 1968 │\n",
       "│ BMW    ┆ M1     ┆ 1978 │\n",
       "│ Toyota ┆ Celica ┆ 1975 │\n",
       "└────────┴────────┴──────┘"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# connectorx is required\n",
    "uri = f'postgres://{postgres_user}:{postgres_pass}@localhost:5432/postgres' \n",
    "query = 'SELECT * FROM sandbox.cars'\n",
    "df = pl.read_database_uri(query, uri)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>brand</th><th>model</th><th>year</th></tr><tr><td>str</td><td>str</td><td>i32</td></tr></thead><tbody><tr><td>&quot;Volvo&quot;</td><td>&quot;p1800&quot;</td><td>1968</td></tr><tr><td>&quot;BMW&quot;</td><td>&quot;M1&quot;</td><td>1978</td></tr><tr><td>&quot;Toyota&quot;</td><td>&quot;Celica&quot;</td><td>1975</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 3)\n",
       "┌────────┬────────┬──────┐\n",
       "│ brand  ┆ model  ┆ year │\n",
       "│ ---    ┆ ---    ┆ ---  │\n",
       "│ str    ┆ str    ┆ i32  │\n",
       "╞════════╪════════╪══════╡\n",
       "│ Volvo  ┆ p1800  ┆ 1968 │\n",
       "│ BMW    ┆ M1     ┆ 1978 │\n",
       "│ Toyota ┆ Celica ┆ 1975 │\n",
       "└────────┴────────┴──────┘"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pip install adbc-driver-postgresql pyarrow\n",
    "df = pl.read_database_uri(query, uri, engine='adbc')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>brand</th><th>model</th><th>year</th></tr><tr><td>str</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;Volvo&quot;</td><td>&quot;p1800&quot;</td><td>1968</td></tr><tr><td>&quot;BMW&quot;</td><td>&quot;M1&quot;</td><td>1978</td></tr><tr><td>&quot;Toyota&quot;</td><td>&quot;Celica&quot;</td><td>1975</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 3)\n",
       "┌────────┬────────┬──────┐\n",
       "│ brand  ┆ model  ┆ year │\n",
       "│ ---    ┆ ---    ┆ ---  │\n",
       "│ str    ┆ str    ┆ i64  │\n",
       "╞════════╪════════╪══════╡\n",
       "│ Volvo  ┆ p1800  ┆ 1968 │\n",
       "│ BMW    ┆ M1     ┆ 1978 │\n",
       "│ Toyota ┆ Celica ┆ 1975 │\n",
       "└────────┴────────┴──────┘"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pip install sqlalchemy pg8000 or psycopg2 (default is psycopg2)\n",
    "from sqlalchemy import create_engine\n",
    "\n",
    "con_string = f'postgresql+pg8000://{postgres_user}:{postgres_pass}@localhost:5432/postgres' \n",
    "engine = create_engine(con_string)\n",
    "conn = engine.connect()\n",
    "\n",
    "df = pl.read_database(query, connection=conn)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.write_database(table_name=\"sandbox.cars_output\", connection=uri, engine=\"adbc\", if_table_exists='append') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.write_database(table_name=\"sandbox.cars_output\", connection=con_string, engine=\"sqlalchemy\", if_table_exists='replace')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>brand</th><th>model</th><th>year</th></tr><tr><td>str</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;Volvo&quot;</td><td>&quot;p1800&quot;</td><td>1968</td></tr><tr><td>&quot;BMW&quot;</td><td>&quot;M1&quot;</td><td>1978</td></tr><tr><td>&quot;Toyota&quot;</td><td>&quot;Celica&quot;</td><td>1975</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 3)\n",
       "┌────────┬────────┬──────┐\n",
       "│ brand  ┆ model  ┆ year │\n",
       "│ ---    ┆ ---    ┆ ---  │\n",
       "│ str    ┆ str    ┆ i64  │\n",
       "╞════════╪════════╪══════╡\n",
       "│ Volvo  ┆ p1800  ┆ 1968 │\n",
       "│ BMW    ┆ M1     ┆ 1978 │\n",
       "│ Toyota ┆ Celica ┆ 1975 │\n",
       "└────────┴────────┴──────┘"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_database_uri('select * from sandbox.cars_output', uri, engine='adbc')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
